# 第一题的代码, byTLHX, 2022-12-1
import pandas as pd
import os
import re

# 1)统计每个Mashup包含的Web API个数
pd.set_option("display.unicode.east_asian_width", True)
df_mashup = pd.read_csv("Correted-ProgrammableWeb-dataset-main/data/mashup_nodes_estimator.csv", on_bad_lines='skip',
                        sep='\t', engine='python')

dt_number_of_webAPIs_contained_by_mashup = {}
for name in list(df_mashup['name']):
    dt_number_of_webAPIs_contained_by_mashup[name] = set()

df_mashup_invoking_APIs = pd.read_csv("Correted-ProgrammableWeb-dataset-main/data/m-a_edges.csv", on_bad_lines='skip',
                                      sep='\t', engine='python')

for i in range(len(df_mashup_invoking_APIs.index)):
    series = df_mashup_invoking_APIs.iloc[i]
    mashup_name, api_url = series['source'], series['target']
    dt_number_of_webAPIs_contained_by_mashup[mashup_name].add(api_url)

number_of_webAPIs_contained_by_mashup = []
for mashup_name in dt_number_of_webAPIs_contained_by_mashup.keys():
    count = len(dt_number_of_webAPIs_contained_by_mashup[mashup_name])
    number_of_webAPIs_contained_by_mashup.append([mashup_name, count])

df_number_of_webAPIs_contained_by_mashup = pd.DataFrame(data=number_of_webAPIs_contained_by_mashup,
                                                        columns=['Mashup', '包含 Web API 个数']).sort_values(
    '包含 Web API 个数', ascending=False)

filename = "每个 Mashup包含的 Web API 个数.xlsx"
if not os.path.exists("firstQuestion_answer_data/" + filename):
    df_number_of_webAPIs_contained_by_mashup.to_excel("firstQuestion_answer_data/" + filename, sheet_name=filename,
                                                      index=False)
else:
    print(filename + " 已存在!")
# -------------------------------------------------------------------

# 2)统计每个Web API被使用的次数
df_API = pd.read_csv("Correted-ProgrammableWeb-dataset-main/data/api_nodes_estimator.csv", on_bad_lines='skip',
                     sep='\t', engine='python')

dt_number_of_times_webAPI_is_used = {}

for url in list(df_API['url']):
    dt_number_of_times_webAPI_is_used[url] = set()

for i in range(len(df_mashup_invoking_APIs.index)):
    series = df_mashup_invoking_APIs.iloc[i]
    mashup_name, api_url = series['source'], series['target']
    if api_url not in dt_number_of_times_webAPI_is_used.keys():
        dt_number_of_times_webAPI_is_used[api_url] = {mashup_name}
    else:
        dt_number_of_times_webAPI_is_used[api_url].add(mashup_name)

number_of_times_webAPI_is_used = []
for api_url in dt_number_of_times_webAPI_is_used.keys():
    count = len(dt_number_of_times_webAPI_is_used[api_url])
    number_of_times_webAPI_is_used.append([api_url, count])

df_number_of_times_webAPI_is_used = pd.DataFrame(data=number_of_times_webAPI_is_used,
                                                 columns=['API', '被使用次数']).sort_values(
    '被使用次数', ascending=False)

filename = "每个 Web API 被使用的次数.xlsx"
if not os.path.exists("firstQuestion_answer_data/" + filename):
    df_number_of_times_webAPI_is_used.to_excel("firstQuestion_answer_data/" + filename, sheet_name=filename,
                                               index=False)
else:
    print(filename + " 已存在!")
# -------------------------------------------------------------------

# 3)统计 Web API 提供商（URL网址）发布 Web API 的个数
pt_api = r"\"url\": \"(.*?)\\"
pt_provider_url = r"visit_url.*?(https?://.*?)/"
dt_number_of_WebAPI_released_by_provider = {}
filepath = "Correted-ProgrammableWeb-dataset-main/data/raw/accessibility/api_accessibility/"
fileSuffix = ['1.txt', '2.txt', '3.txt', '4.txt', '5.txt']
for suffix in fileSuffix:
    file = open(filepath + "api_version_accessbiliby-" + suffix, mode="r")
    api_version_accessbiliby = file.read().split("version_title")[1:]
    for data in api_version_accessbiliby:
        api = re.search(pt_api, data).group(1)
        provider_url = re.findall(pt_provider_url, data)
        for url in provider_url:
            if url not in dt_number_of_WebAPI_released_by_provider.keys():
                dt_number_of_WebAPI_released_by_provider[url] = {api}
            else:
                dt_number_of_WebAPI_released_by_provider[url].add(api)
    file.close()

number_of_WebAPI_released_by_provider = []
for provider in dt_number_of_WebAPI_released_by_provider.keys():
    count = len(dt_number_of_WebAPI_released_by_provider[provider])
    number_of_WebAPI_released_by_provider.append([provider, count])

df_number_of_WebAPI_released_by_provider = pd.DataFrame(data=number_of_WebAPI_released_by_provider,
                                                        columns=['Web API 提供商(URL 网址)',
                                                                 '发布 Web API 的个数']).sort_values(
    '发布 Web API 的个数', ascending=False)

filename = "WebAPI提供商(URL网址)发布WebAPI个数.xlsx"
if not os.path.exists("firstQuestion_answer_data/" + filename):
    df_number_of_WebAPI_released_by_provider.to_excel("firstQuestion_answer_data/" + filename, sheet_name=filename,
                                                      index=False)
else:
    print(filename + " 已存在!")
